# MIT License
# Python 3.1.2

import datetime
import time
import sys
import string
import re
import socket
import operator
import os

# Enumerate from the day symbols to the day numbers. 
# The numbers of the day are just an offset by which we subtract
# from the current date in order to obtain the previous week beginning 
# The origin of the offsets is Saturday.
DAY_ENUMERATE={0:0,'Sun':1,'Mon':2,'Tue':3,'Wed':4,'Thu':5,'Fri':6,'Sat':7}
            
#################################################
#                AUXILERY CLASSES               #
#################################################    

#   htmlFields constructor takes 4 parameters that are considered constant throughout the HTML file
#   These fields are the student name the log file and the start date of the week with the end date
#   It is a Class describing the HTML fields that must be completed on the HTML document
#   This class acts as a way to map each element on the html with the variable of interest.
class htmlFields:
    
    def __init__(self, name, file,startDate,stopDate):
    # Initializing all fields
        self.STUDENT_NAME = name
        self.FILE_PROCESSED = file
        self.WEEK_START = startDate
        self.WEEK_STOP = stopDate
        #defined as integer object look at integer class for more info
        self.WEEK_ENTRIES = integer(0)
        self.OTHER_ENTRIES = integer(0)
        self.IGNORED_ENTRIES = integer(0)
        self.TOTAL_ENTRIES = integer(0)
        self.DISTINCT_HOSTS = integer(0)
        self.DISTINCT_HOSTS_LIST = list()
        self.DISTINCT_HOSTS_LIST_WEEK = list()
        self.DISTINCT_PAGES_LIST_WEEK = list()
        self.NOT_RESOLVED_HOSTS = integer(0)
        #Week's totals
        self.WEEK_ERRORS=[0,0,0,0,0,0,0,0]
        self.WEEK_INFORM=[0,0,0,0,0,0,0,0]  
        self.WEEK_REDIRECTS=[0,0,0,0,0,0,0,0]   
        self.WEEK_SERVER_ERROR=[0,0,0,0,0,0,0,0]  
        self.WEEK_BYTES=[0,0,0,0,0,0,0,0]    
        self.WEEK_HITS=[0,0,0,0,0,0,0,0]  
        self.PERCENT_CS=integer(0.0)
        self.PERCENT_FIT=integer(0.0)
        self.PERCENT_EDU=integer(0.0)
        # Prase table in the form of dictionary. Provides with fast maping and access
        self.PARSE_TABLE={"$_ERROR":self.WEEK_ERRORS,"$_INFORM":self.WEEK_INFORM,
        "$_REDIR":self.WEEK_REDIRECTS,"$_SERVER_ERROR":self.WEEK_SERVER_ERROR,
        "$_BYTES":self.WEEK_BYTES,"$_HITS":self.WEEK_HITS,"$_NAME":self.STUDENT_NAME,
        "$_FILE_PROCESSED":self.FILE_PROCESSED,"$_WEEK_START":self.WEEK_START,"$_WEEK_STOP":self.WEEK_STOP,
        "$_WEEK_ENTRIES":self.WEEK_ENTRIES,"$_NOT_WEEK_ENTRIES":self.OTHER_ENTRIES,"$_IGNORED_ENTRIES":self.IGNORED_ENTRIES,
        "$_TOTAL_ENTRIES":self.TOTAL_ENTRIES,"$_DISTINCT_HOST":self.DISTINCT_HOSTS,"$_RESOLVED_HOST":self.NOT_RESOLVED_HOSTS,
        "$_PERCENT_CS":self.PERCENT_CS,"$_PERCENT_FIT":self.PERCENT_FIT,"$_PERCENT_EDU":self.PERCENT_EDU,
        "$_TOP_HOST_NAME":self.DISTINCT_HOSTS_LIST_WEEK, "$_TOP_PAGE_NAME":self.DISTINCT_PAGES_LIST_WEEK}
    
    # Prints to the standard output the html after parsing all fields
    def printHTML(self):
        # The html template should be named index.html and be placed in the same location
        with open(os.path.join(os.path.dirname(__file__),"index.html"), 'r+') as fileHandler:
            # Go over every line in the file and parse it
            for line in fileHandler:
                output=line
                # For every line in the html go over the parse table
                for j in self.PARSE_TABLE:
                    # If a symbol (or index) in the parse table matches with the line
                    if j in output:
                        # We determine what kind of symbol it is. 
                        # Three categories exist, One of them is an array the next type is a list 
                        # and finally is a variable
                        if re.match(".+\[[A-Z0-9]+\]",output):
                            # If it is an array, find to which index the symbol points (For this 
                            # case it is just days of the week)
                            for i in DAY_ENUMERATE:
                                # Try to match the index with the symbols index
                                index="["+str(i).upper()+"]"
                                if index in output:
                                    # We then save the output to be the output after we replace the symbol with the value
                                    weeklyValue=self.PARSE_TABLE[j]
                                    output = output.replace(j+index,str(weeklyValue[DAY_ENUMERATE[i]]))
                                    break
                            #potential optimization is to break here if we know for sure that there will be only one variable per row
                        elif "$_LIST" in output:
                            # If it is a list (Somewhat similar to linked list but in Python its treated like an array
                            # Then we try to find the top 10 something for this HTML file. For that reason we only print 10
                            # elements of the list which we already have sorted before
                            if len(self.PARSE_TABLE[j])>0:
                                # The syntax of the javascript provided by Google is as follows for every element
                                dataArray="['Host', 'Requests']"
                                # The limit of hosts to be displayed
                                topLimit=10
                                for k in self.PARSE_TABLE[j]:
                                    # The output is of the form [hostname, request count]
                                    # Every element in the list is saved as an object and that object has two 
                                    # member variables. One of the member variables is the name and the other the 
                                    # number of weekly requests. 
                                    dataArray+=",\n['"+k.getName()+"',"+str(k.getCount())+"]"
                                    topLimit-=1
                                    if(topLimit==0):
                                        break
                                # The symbol names are commented in the case of lack of data to display. In that case
                                # there will be the output "No columns on the table"
                                symbolName="//"+j+"[$_LIST]"
                                output=output.replace(symbolName,dataArray)
                        else:
                            # If it just a variable than things are easy. If it is an object we need to use
                            # the getter methods to retrieve it. If it is a python type than we just output it.
                            if isinstance(self.PARSE_TABLE[j],integer):
                                output = output.replace(j,str(self.PARSE_TABLE[j].getOutput()))
                            else:
                                output = output.replace(j,str(self.PARSE_TABLE[j]))
                            #same as above we could break here for optimization if we know for sure that there will be only one variable per row
                print(output)
                
# This class is used to simplify the list and create one list but of multiple objects. 
# the address class describes a host and the number of requests the host received over a week.                
class address:
    # constructor with the url and an initiali count of 1
    def __init__(self, url):
        self.NAME=url
        self.COUNT=1
    # Self incrementing function which is called each time the host is found
    def plus(self):
        self.COUNT+=1
    # Returns the name of the host 
    def getName(self):
        return self.NAME
    # Returns the number of requests
    def getCount(self):
        return self.COUNT

# class integer used. I was confused by the behaviour of python and the output
# after asking a question at stackoverflow.com it was made more obvious that the
# value is rather kept as a value and not a reference or address. That is because
# WEEK_ENTRIES if defined as =0 then it is a primitive type and a VALUE. Rather I created
# an object whose address I maintain through the dictionary.
# related thread http://stackoverflow.com/questions/19287143/unexpected-python-behavior-with-dictionary-and-class
class integer:
    # The constructor initiates the object with an initial value
    def __init__(self, integer):
        self.value=integer
    # Increase the integer value by one
    def plus(self):
        self.value=self.value+1
    # Getter method for the value
    def getOutput(self):
        return self.value    
    # sets a value in the number, similar to the constructor
    def setValue(self, number):
        self.value=number

#################################################
#            AUXILIARY   METHODS                #
#################################################

# Goes over the list and tries to match the host. If it fails to match it 
# returns false and adds the host to the list. If it does match it, it increases
# the host count
def findDistinct(list,host):
    for i in list:
        if host==i.getName():
            i.plus()
            return 0
    # initiates an object of address with host and adds it to the list        
    list.append(address(host))
    return 1
    
# Prints instructions    
def usage():
    print("\nThere was an error on your command line arguments")
    print("example of use: log.py /logfile.txt 10/Dec/2010")
    print("\nThe date is an optional argument and if not explicity stated the")
    print("current date will be used. Also make sure the log file exists.")

# Parses a log entry to classify it and alter the html variables as expected     
def parseLogEntry(html, line):
    # Increases the total entries
    html.TOTAL_ENTRIES.plus()
    
    # regular expression to match log entry used from http://effbot.org/zone/re-common-log-format.htm
    logFormat = re.compile(
    '([^ ]*) ([^ ]*) ([^ ]*) \[([^]]*)\] "([^"]*)" ([^ ]*) ([^ ]*)'
    ) 
    # The first check to find if entry is illegal or not is the format
    if not logFormat.match(line):
        html.IGNORED_ENTRIES.plus()
        return
    # If the format is correct retrieve the data. 
    entry=line.split(" ")
    # The host name
    host=entry[0]
    dateString=entry[3].replace("[","")
    # entry[4] is the time-zone which after further talk with Dr. Stansifer should be ignored
    # %z value = +- integer of format HHMM
    # entry[5] is the GET statement
    # elements[7] is the HTTP/1.x statement
    requestDirectory=entry[6]
    status=entry[8]
    bytes=entry[9]
    # To ensure the entry is not illegal we form check of the date.
    try:
        date=datetime.datetime.strptime(dateString, "%d/%b/%Y:%H:%M:%S")
    except ValueError:
        html.IGNORED_ENTRIES.plus()
        return
    # If the host name is an IP try to match it with the pattern. 
    if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$",host):
        # There was no specification from Dr. Stansifer about socket timeout.
        # The current method ensures that the hostname doesn't resolve but takes few seconds per entry
        # We can decrease the timeout but risk on the socket not been resolved accurately. 
        try:
            resolvedHost=socket.gethostbyaddr(host)
            host=resolvedHost[0]
            # If the host name is resolved then try to find if its a distinct host
            if findDistinct(html.DISTINCT_HOSTS_LIST,host):
                html.DISTINCT_HOSTS.plus()
        except socket.herror:
            html.NOT_RESOLVED_HOSTS.plus()
    else:
        # In case the hostname is already a domain then try to find if its distinct
        # If it is distinct increase distinct count
        if findDistinct(html.DISTINCT_HOSTS_LIST,host):
            html.DISTINCT_HOSTS.plus()
    # If the entry is within the dates of the week examined perform some observations    
    if html.WEEK_START<date.date() and date.date()<html.WEEK_STOP:
        # increase the week entry count
        html.WEEK_ENTRIES.plus()
        # If the domain name is a .edu domain name it is also a fit.edu and cs.fit.edu domain
        if re.match("[a-zA-Z0-9\.\-]*edu",host):
            # The PERCENT_EDU is firstly used as a counter and after all calculations the percentage
            # is computed
            html.PERCENT_EDU.plus()
            # If the domain name is an fit.edu domain it is also a cs.fit.edu domain
            if re.match("[a-zA-Z0-9\.\-]*fit\.edu",host):
                html.PERCENT_FIT.plus()
                if re.match("[a-zA-Z0-9\.\-]*cs\.fit\.edu",host):
                    html.PERCENT_CS.plus()
        
        # Statements to determine the status of the request
        if status[0]=="1":
            html.WEEK_INFORM[DAY_ENUMERATE[date.strftime("%a")]]+=1
            html.WEEK_INFORM[0]+=1
        elif status[0]=="2":
            html.WEEK_HITS[DAY_ENUMERATE[date.strftime("%a")]]+=1
            html.WEEK_HITS[0]+=1
            # For the number of bytes we only count the actual hits as specified by the assignment
            html.WEEK_BYTES[DAY_ENUMERATE[date.strftime("%a")]]+=int(bytes)
            html.WEEK_BYTES[0]+=int(bytes)
            # NOT TO BE CONFUSED WITH if findDistinct(html.DISTINCT_HOSTS_LIST,host):
            # This function now determines how distinct the weekly host requests are 
            findDistinct(html.DISTINCT_HOSTS_LIST_WEEK,host)
            # It does the same for the pages
            findDistinct(html.DISTINCT_PAGES_LIST_WEEK,requestDirectory)
        elif status[0]=="3":
            html.WEEK_REDIRECTS[DAY_ENUMERATE[date.strftime("%a")]]+=1
            html.WEEK_REDIRECTS[0]+=1
        elif status[0]=="4":
            html.WEEK_ERRORS[DAY_ENUMERATE[date.strftime("%a")]]+=1
            html.WEEK_ERRORS[0]+=1
        elif status[0]=="5":
            html.WEEK_SERVER_ERROR[DAY_ENUMERATE[date.strftime("%a")]]+=1
            html.WEEK_SERVER_ERROR[0]+=1
    else:
        # For entries outside the week
        html.OTHER_ENTRIES.plus()
        
#################################################
#               MAIN   METHOD                   #
#################################################

# Main method invoked first
def main():
    logFile = None
    if len(sys.argv) < 2:
        # If There is no file argument exit with error message
        usage()
        sys.exit()
    else:
        try:
            with open(sys.argv[1]):
                logFile=sys.argv[1]
        except IOError:
            # If the file can't be opened, display error message
            usage()
            sys.exit()
        # If there is a date argument
        if len(sys.argv)>2:
                week = datetime.datetime.strptime(sys.argv[2], "%d/%b/%Y").date()
        else:
                # Otherwise use today's date
                week = datetime.datetime.now().date()
    # Get today's date to find the past week
    todayName = week.strftime("%a")
    # The past week is the offset from todays day to saturday
    stopDate = week - datetime.timedelta(DAY_ENUMERATE[todayName])
    # The stop day -6 is a sunday and its the begining of the week
    startDate = stopDate - datetime.timedelta(6)
    # Create an HTML instance of my name start and stop date.
    htmlFile=htmlFields("Iordanis Fostiropoulos",logFile,startDate,stopDate)
    # Open log and parse every entry
    with open(os.path.join(os.path.dirname(__file__),logFile), 'r') as fileHandler:
        for line in fileHandler:
            parseLogEntry(htmlFile, line.rstrip())
    # After parsing every entry, calculate the percentage only if there is at least one disinct host
    # To avoid division by zero
    if htmlFile.DISTINCT_HOSTS.getOutput()!=0:
        htmlFile.PERCENT_CS.setValue(round(htmlFile.PERCENT_CS.getOutput()/htmlFile.DISTINCT_HOSTS.getOutput(),2))
        htmlFile.PERCENT_FIT.setValue(round(htmlFile.PERCENT_FIT.getOutput()/htmlFile.DISTINCT_HOSTS.getOutput(),2))
        htmlFile.PERCENT_EDU.setValue(round(htmlFile.PERCENT_EDU.getOutput()/htmlFile.DISTINCT_HOSTS.getOutput(),2))
    # The assignment had resources for selection sort but didn't specify the way to sort. The build-in sort is used
    # sorting is done in reverse order and the COUNT variable as a key
    htmlFile.DISTINCT_HOSTS_LIST_WEEK.sort(key=operator.attrgetter('COUNT'),reverse=True)
    htmlFile.DISTINCT_PAGES_LIST_WEEK.sort(key=operator.attrgetter('COUNT'),reverse=True)
    # The html file is printed on the standard output
    htmlFile.printHTML()

main()
